{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PPO agent using stable-baselines3\n",
    "\n",
    "In this notebook, we will look at policy optimization using PPO. We will not be writing our own algorithms. Rather, we will use a fork of [OpenAI Baselines](https://github.com/openai/baselines). The version we will use is still in beta but good enough for your tutorials. THe github repository for baselines-3 : https://github.com/DLR-RM/stable-baselines3\n",
    "\n",
    "\n",
    "The code below follows the [getting started tutorial](https://colab.research.google.com/github/Stable-Baselines-Team/rl-colab-notebooks/blob/sb3/stable_baselines_getting_started.ipynb) that comes with this library. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.ppo.policies import MlpPolicy\n",
    "\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "import os\n",
    "import io\n",
    "import base64\n",
    "import time\n",
    "import glob\n",
    "from IPython.display import HTML\n",
    "\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Environment - CartPole \n",
    "\n",
    "We can use the setup here to run on any environment which has state as a single vector and actions are discrete. We will build it on Cart Pole and they try to run this on many other environments like Atari games and others."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_env(env_name, seed=None):\n",
    "    # remove time limit wrapper from environment\n",
    "    env = gym.make(env_name)\n",
    "    if seed is not None:\n",
    "        env.seed(seed)\n",
    "    return env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAATPklEQVR4nO3db4xd9Z3f8ffHf0mANngZqNc2a5Y11ULUNaupNypVlw10oaRaJw9SOVIjHhA5DxwpqKuksCt1E6mWttUmqSI1kUiD1kqzMZYSisVmu8u6idKoFDAsENvg4AQvntjYxgl/Q4zt+fbBHMrFnvFcz5+Mf3PfL+nqnvs9v3PP94eGj49/PnduqgpJUjsWzHUDkqRzY3BLUmMMbklqjMEtSY0xuCWpMQa3JDVm1oI7yS1J9ibZl+TO2TqPJA2azMZ93EkWAj8E/iUwAjwKfKSq9sz4ySRpwMzWFfc6YF9V/biq3gS2Autn6VySNFAWzdL7rgAO9LweAX5nosGXXnpprV69epZakaT27N+/nxdffDHj7Zut4B7vZO9Yk0myEdgIcMUVV7Bz585ZakWS2jM8PDzhvtlaKhkBVvW8Xgkc7B1QVXdX1XBVDQ8NDc1SG5I0/8xWcD8KrElyZZIlwAZg+yydS5IGyqwslVTVySSfAP4aWAjcU1W7Z+NckjRoZmuNm6r6NvDt2Xp/SRpUfnJSkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjpvXVZUn2A68Cp4CTVTWcZBlwL7Aa2A/8m6r62fTalCS9ZSauuH+vqtZW1XD3+k5gR1WtAXZ0ryVJM2Q2lkrWA1u67S3AB2fhHJI0sKYb3AX8TZLHkmzsapdX1SGA7vmyaZ5DktRjWmvcwPVVdTDJZcCDSZ7p98Au6DcCXHHFFdNsQ5IGx7SuuKvqYPd8BLgPWAccTrIcoHs+MsGxd1fVcFUNDw0NTacNSRooUw7uJBcmufitbeD3gV3AduC2bthtwP3TbVKS9LbpLJVcDtyX5K33+Yuq+p9JHgW2JbkdeB748PTblCS9ZcrBXVU/Bn5rnPox4MbpNCVJmpifnJSkxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaM2lwJ7knyZEku3pqy5I8mOTZ7vmSnn13JdmXZG+Sm2ercUkaVP1ccf85cMtptTuBHVW1BtjRvSbJNcAG4NrumC8lWThj3UqSJg/uqvoe8NPTyuuBLd32FuCDPfWtVXW8qp4D9gHrZqZVSRJMfY378qo6BNA9X9bVVwAHesaNdLUzJNmYZGeSnUePHp1iG5I0eGb6HyczTq3GG1hVd1fVcFUNDw0NzXAbkjR/TTW4DydZDtA9H+nqI8CqnnErgYNTb0+SdLqpBvd24LZu+zbg/p76hiRLk1wJrAEemV6LkqReiyYbkOQbwA3ApUlGgD8B/hTYluR24HngwwBVtTvJNmAPcBLYVFWnZql3SRpIkwZ3VX1kgl03TjB+M7B5Ok1JkibmJyclqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDVm0uBOck+SI0l29dQ+k+QnSZ7oHrf27Lsryb4ke5PcPFuNS9Kg6ueK+8+BW8apf6Gq1naPbwMkuQbYAFzbHfOlJAtnqllJUh/BXVXfA37a5/utB7ZW1fGqeg7YB6ybRn+SpNNMZ437E0me6pZSLulqK4ADPWNGutoZkmxMsjPJzqNHj06jDUkaLFMN7i8DVwFrgUPA57p6xhlb471BVd1dVcNVNTw0NDTFNiRp8EwpuKvqcFWdqqpR4Cu8vRwyAqzqGboSODi9FiVJvaYU3EmW97z8EPDWHSfbgQ1Jlia5ElgDPDK9FiVJvRZNNiDJN4AbgEuTjAB/AtyQZC1jyyD7gY8DVNXuJNuAPcBJYFNVnZqVziVpQE0a3FX1kXHKXz3L+M3A5uk0JUmamJ+clKTGGNyS1BiDW5IaY3BLUmMMbklqjMGtgXbyF6/x+tG/n+s2pHMy6e2A0nxy8LEHeP3wj/7/6xNvvMqCRYv5x3/waZLxfmODdP4xuDVQ3jh2gFdG9ryjduHlvz5H3UhT41KJJDXG4JakxhjcktQYg1sDZeja3+P0Xxv/xrERXjv0w7lpSJoCg1sDZcmF7znj6z5GT77JqTffmJN+pKkwuCWpMQa3JDXG4JakxhjcktQYg1uSGjNpcCdZleQ7SZ5OsjvJJ7v6siQPJnm2e76k55i7kuxLsjfJzbM5AUkaNP1ccZ8E/rCqfhN4H7ApyTXAncCOqloD7Ohe0+3bAFwL3AJ8KcnC2WhekgbRpMFdVYeq6vFu+1XgaWAFsB7Y0g3bAnyw214PbK2q41X1HLAPWDfDfUvSwDqnNe4kq4HrgIeBy6vqEIyFO3BZN2wFcKDnsJGudvp7bUyyM8nOo0ePTqF1SRpMfQd3kouAbwJ3VNUrZxs6Tq3OKFTdXVXDVTU8NDTUbxuSNPD6Cu4kixkL7a9X1be68uEky7v9y4EjXX0EWNVz+Erg4My0K03PgkVLWHLhJWfUf/HyEarOuL6Qzkv93FUS4KvA01X1+Z5d24Hbuu3bgPt76huSLE1yJbAGeGTmWpambslFy/gHq649o/7iM98Hg1uN6OcbcK4HPgr8IMkTXe2PgD8FtiW5HXge+DBAVe1Osg3Yw9gdKZuq6tRMNy5Jg2rS4K6q7zP+ujXAjRMcsxnYPI2+JEkT8JOTktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3BIwevJN3nz9Z3PdhtQXg1sD51eu/mdk4eJ31E68/jNeGdkzRx1J58bg1sBZevGljP3uNKlNBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY3p58uCVyX5TpKnk+xO8smu/pkkP0nyRPe4teeYu5LsS7I3yc2zOQFJGjT9fFnwSeAPq+rxJBcDjyV5sNv3har6s97BSa4BNgDXAr8K/G2Sq/3CYEmaGZNecVfVoap6vNt+FXgaWHGWQ9YDW6vqeFU9B+wD1s1Es5Kkc1zjTrIauA54uCt9IslTSe5JcklXWwEc6DlshLMHvfRLtXDpu7nkqn96Rv3Y3v/D6MkTc9CRdG76Du4kFwHfBO6oqleALwNXAWuBQ8Dn3ho6zuE1zvttTLIzyc6jR4+ea9/SlC1YuIglFy07o3781RepGp2DjqRz01dwJ1nMWGh/vaq+BVBVh6vqVI39pH+Ft5dDRoBVPYevBA6e/p5VdXdVDVfV8NDQ0HTmIEkDpZ+7SgJ8FXi6qj7fU1/eM+xDwK5uezuwIcnSJFcCa4BHZq5lSRps/dxVcj3wUeAHSZ7oan8EfCTJWsaWQfYDHweoqt1JtgF7GLsjZZN3lEjSzJk0uKvq+4y/bv3tsxyzGdg8jb4kSRPwk5OS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3BtLid/9DsmDhO2o1eooTP395jjqS+mdwayAt+411LFx64Ttqp46/zk/3PTzBEdL5w+CWpMYY3JLUGINbkhpjcEtSYwxuSWpMP7/WVWrC6Ogod9xxBwcOHJh07OKFYdPvLuOipe+8JfDerffyvf94T1/n27RpEzfddNOUepWmw+DWvLJjxw727Nkz6bgLlizi9t/ZwJLFl1A19hfPRQve5JlnnuF/PPBYX+f6wAc+MK1epakyuDWwXj5xKU8eXc+bdQEA/+iC5zhZj89xV9LkXOPWQKoKz//8N3lj9GJO1WJO1WJ+8sYa/v7n18x1a9KkDG4NpFEW8sIvfu20ajhVi+ekH+lc9PNlwRckeSTJk0l2J/lsV1+W5MEkz3bPl/Qcc1eSfUn2Jrl5NicgTcWCnOKKdz/zjlo4xdIFr89RR1L/+rniPg68v6p+C1gL3JLkfcCdwI6qWgPs6F6T5BpgA3AtcAvwpSQLx3tjaa6cOHGSPbv+kmVLDrFo9EVefHE/i1/7Dr971RHevdSrbp3f+vmy4AJe614u7h4FrAdu6OpbgO8C/76rb62q48BzSfYB64CHJjrHiRMneOGFF6Y2A6kzOjrKyZMn+xtbxV8/9ChHXvw0x155g//91PNAkW5fP1555RV/bjVrTpw4MeG+vu4q6a6YHwN+A/ivVfVwksur6hBAVR1Kclk3fAXwf3sOH+lqEzp27Bhf+9rX+mlFmlBV8fLL/f9a1v0vvMT+F15653ucw/keeughTp06dQ5HSP07duzYhPv6Cu6qOgWsTfIe4L4k7z3L8Iz3FmcMSjYCGwGuuOIKPvWpT/XTijSh0dFRtmzZwuHDh38p57v55pv52Mc+9ks5lwbPvffeO+G+c7qrpKpeYmxJ5BbgcJLlAN3zkW7YCLCq57CVwMFx3uvuqhququGhoaFzaUOSBlo/d5UMdVfaJHkXcBPwDLAduK0bdhtwf7e9HdiQZGmSK4E1wCMz3LckDax+lkqWA1u6de4FwLaqeiDJQ8C2JLcDzwMfBqiq3Um2AXuAk8CmbqlFkjQD+rmr5CngunHqx4AbJzhmM7B52t1Jks7gJyclqTEGtyQ1xt8OqHnlxhtv5Oqrr/6lnGv16tW/lPNIpzO4NW8sWLCAL37xi3PdhjTrXCqRpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY3p58uCL0jySJInk+xO8tmu/pkkP0nyRPe4teeYu5LsS7I3yc2zOQFJGjT9/D7u48D7q+q1JIuB7yf5q27fF6rqz3oHJ7kG2ABcC/wq8LdJrvYLgyVpZkx6xV1jXuteLu4edZZD1gNbq+p4VT0H7APWTbtTSRLQ5xp3koVJngCOAA9W1cPdrk8keSrJPUku6WorgAM9h490NUnSDOgruKvqVFWtBVYC65K8F/gycBWwFjgEfK4bnvHe4vRCko1JdibZefTo0Sm0LkmD6ZzuKqmql4DvArdU1eEu0EeBr/D2csgIsKrnsJXAwXHe6+6qGq6q4aGhoan0LkkDqZ+7SoaSvKfbfhdwE/BMkuU9wz4E7Oq2twMbkixNciWwBnhkRruWpAHWz10ly4EtSRYyFvTbquqBJF9LspaxZZD9wMcBqmp3km3AHuAksMk7SiRp5kwa3FX1FHDdOPWPnuWYzcDm6bUmSRqPn5yUpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNSVXNdQ8kOQq8Drw4173MgktxXq2Zr3NzXm35taoaGm/HeRHcAEl2VtXwXPcx05xXe+br3JzX/OFSiSQ1xuCWpMacT8F991w3MEucV3vm69yc1zxx3qxxS5L6cz5dcUuS+jDnwZ3kliR7k+xLcudc93OuktyT5EiSXT21ZUkeTPJs93xJz767urnuTXLz3HQ9uSSrknwnydNJdif5ZFdvem5JLkjySJInu3l9tqs3Pa+3JFmY5O+SPNC9ni/z2p/kB0meSLKzq82LuU1JVc3ZA1gI/Aj4dWAJ8CRwzVz2NIU5/Avgt4FdPbX/DNzZbd8J/Kdu+5pujkuBK7u5L5zrOUwwr+XAb3fbFwM/7Ppvem5AgIu67cXAw8D7Wp9Xz/z+HfAXwAPz5Wex63c/cOlptXkxt6k85vqKex2wr6p+XFVvAluB9XPc0zmpqu8BPz2tvB7Y0m1vAT7YU99aVcer6jlgH2P/Dc47VXWoqh7vtl8FngZW0Pjcasxr3cvF3aNofF4ASVYCHwD+W0+5+XmdxXye21nNdXCvAA70vB7paq27vKoOwVgAApd19Sbnm2Q1cB1jV6fNz61bTngCOAI8WFXzYl7AfwE+DYz21ObDvGDsD9e/SfJYko1dbb7M7ZwtmuPzZ5zafL7Npbn5JrkI+CZwR1W9kow3hbGh49TOy7lV1SlgbZL3APclee9ZhjcxryT/GjhSVY8luaGfQ8apnXfz6nF9VR1MchnwYJJnzjK2tbmds7m+4h4BVvW8XgkcnKNeZtLhJMsBuucjXb2p+SZZzFhof72qvtWV58XcAKrqJeC7wC20P6/rgT9Isp+xJcf3J/nvtD8vAKrqYPd8BLiPsaWPeTG3qZjr4H4UWJPkyiRLgA3A9jnuaSZsB27rtm8D7u+pb0iyNMmVwBrgkTnob1IZu7T+KvB0VX2+Z1fTc0sy1F1pk+RdwE3AMzQ+r6q6q6pWVtVqxv4/+l9V9W9pfF4ASS5McvFb28DvA7uYB3Obsrn+11HgVsbuWPgR8Mdz3c8U+v8GcAg4wdif9LcDvwLsAJ7tnpf1jP/jbq57gX811/2fZV7/nLG/Xj4FPNE9bm19bsA/Af6um9cu4D909abnddocb+Dtu0qanxdjd5092T12v5UT82FuU334yUlJasxcL5VIks6RwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmP+Hy0kw4eSz/MPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env_name = 'CartPole-v1'\n",
    "\n",
    "env = make_env(env_name)\n",
    "env.reset()\n",
    "plt.imshow(env.render(\"rgb_array\"))\n",
    "state_shape, n_actions = env.observation_space.shape, env.action_space.n\n",
    "state_dim = state_shape[0]\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build Actor Critic Network\n",
    "\n",
    "We will build two simple networks that take in state. One network produces logits for the action probabilities. 2nd network produces the Value of the state. The observation space and action space is as given below for CartPole\n",
    "\n",
    "    Observation:\n",
    "        Type: Box(4)\n",
    "        Num     Observation               Min                     Max\n",
    "        0       Cart Position             -4.8                    4.8\n",
    "        1       Cart Velocity             -Inf                    Inf\n",
    "        2       Pole Angle                -0.418 rad (-24 deg)    0.418 rad (24 deg)\n",
    "        3       Pole Angular Velocity     -Inf                    Inf\n",
    "    Actions:\n",
    "        Type: Discrete(2)\n",
    "        Num   Action\n",
    "        0     Push cart to the left\n",
    "        1     Push cart to the right\n",
    "        \n",
    "\n",
    "Each model will be a simple one with 1 hidden layer with Relu activation and final layer being logits (for policy/actor network) and value of the state for the Critic Network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = PPO(MlpPolicy, env, verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Untrained Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_reward:9.80 +/- 0.57\n"
     ]
    }
   ],
   "source": [
    "mean_reward, std_reward = evaluate_policy(model, env, n_eval_episodes=100)\n",
    "\n",
    "print(f\"mean_reward:{mean_reward:.2f} +/- {std_reward:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x1ae8a90a160>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the agent for 30000 steps\n",
    "model.learn(total_timesteps=30000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate Trained Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_reward:500.00 +/- 0.00\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the trained agent\n",
    "mean_reward, std_reward = evaluate_policy(model, env, n_eval_episodes=100)\n",
    "\n",
    "print(f\"mean_reward:{mean_reward:.2f} +/- {std_reward:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Trained Agent Video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import base64\n",
    "from pathlib import Path\n",
    "\n",
    "from IPython import display as ipythondisplay\n",
    "\n",
    "def show_videos(video_path='', prefix=''):\n",
    "  \"\"\"\n",
    "  Taken from https://github.com/eleurent/highway-env\n",
    "\n",
    "  :param video_path: (str) Path to the folder containing videos\n",
    "  :param prefix: (str) Filter the video, showing only the only starting with this prefix\n",
    "  \"\"\"\n",
    "  html = []\n",
    "  for mp4 in Path(video_path).glob(\"{}*.mp4\".format(prefix)):\n",
    "      video_b64 = base64.b64encode(mp4.read_bytes())\n",
    "      html.append('''<video alt=\"{}\" autoplay \n",
    "                    loop controls style=\"height: 400px;\">\n",
    "                    <source src=\"data:video/mp4;base64,{}\" type=\"video/mp4\" />\n",
    "                </video>'''.format(mp4, video_b64.decode('ascii')))\n",
    "  ipythondisplay.display(ipythondisplay.HTML(data=\"<br>\".join(html)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from stable_baselines3.common.vec_env import VecVideoRecorder, DummyVecEnv\n",
    "\n",
    "def record_video(env_id, model, video_length=500, prefix='', video_folder='videos/baselines/ppo'):\n",
    "  \"\"\"\n",
    "  :param env_id: (str)\n",
    "  :param model: (RL model)\n",
    "  :param video_length: (int)\n",
    "  :param prefix: (str)\n",
    "  :param video_folder: (str)\n",
    "  \"\"\"\n",
    "  eval_env = DummyVecEnv([lambda: gym.make(env_id)])\n",
    "  # Start the video at step=0 and record 500 steps\n",
    "  eval_env = VecVideoRecorder(eval_env, video_folder=video_folder,\n",
    "                              record_video_trigger=lambda step: step == 0, video_length=video_length,\n",
    "                              name_prefix=prefix)\n",
    "\n",
    "  obs = eval_env.reset()\n",
    "  for _ in range(video_length):\n",
    "    action, _ = model.predict(obs)\n",
    "    obs, _, _, _ = eval_env.step(action)\n",
    "\n",
    "  # Close the video recorder\n",
    "  eval_env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "record_video('CartPole-v1', model, video_length=500, prefix='ppo2-cartpole')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
